import requests
from lxml import etree
from queue import Queue
import threading
import time
#获取数据
'''
def craw(start):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0'
    }
    url = 'https://yz.chsi.com.cn/sch/search.do?'
    param = {
        'start': start
    }
    r = requests.get(url=url, params=param, headers=headers)
    return r.text

#数据解析
def parse(html):
    tree = etree.HTML(html)
    div_list = tree.xpath('//div[@class="sch-list-container"]/div')
    schools = []
    for div in div_list:
        name_data = div.xpath('.//a[@class="name js-yxk-yxmc text-decoration-none"]/text()')[0]
        name_data = name_data.strip()
        href_data = 'https://yz.chsi.com.cn' + \
                    div.xpath('.//a[@class="name js-yxk-yxmc text-decoration-none"]/@href')[0]
        schools.append((name_data, href_data))
    return schools

if __name__ == '__main__':
    for a in range(40):
        start = '0'
        start = str(int(start) + 20)
        schools = parse(craw(start))
        for name, href in schools:
            print(name, href)
        print(f'------{a + 1}------')'''
import requests
from lxml import etree
from queue import Queue
import threading
import time

# 队列
q = Queue()

# 生产者函数：抓取数据并放入队列
def producer(start, q):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0'
    }
    url = 'https://yz.chsi.com.cn/sch/search.do?'
    param = {
        'start': start
    }
    r = requests.get(url=url, params=param, headers=headers)
    if r.status_code == 200:
        q.put(r.text)
    else:
        print(f"Failed to fetch page with start={start}, status code: {r.status_code}")

# 消费者函数：从队列中取出数据并解析
def consumer(q):
    while True:
        html = q.get()
        if html is None:  # 如果队列为空，则退出循环
            q.task_done()
            break
        schools = parse(html)
        for name, href in schools:
            print(name, href)
        q.task_done()  # 通知队列当前任务完成

# 数据解析函数
def parse(html):
    tree = etree.HTML(html)
    div_list = tree.xpath('//div[@class="sch-list-container"]/div')
    schools = []
    for div in div_list:
        name_data = div.xpath('.//a[@class="name js-yxk-yxmc text-decoration-none"]/text()')[0]
        name_data = name_data.strip()
        href_data = 'https://yz.chsi.com.cn' + \
                    div.xpath('.//a[@class="name js-yxk-yxmc text-decoration-none"]/@href')[0]
        schools.append((name_data, href_data))
    return schools

# 主函数
if __name__ == '__main__':
    # 创建生产者线程
    threads_producer = []
    for a in range(0, 800, 20):  # 假设我们抓取0到79页的数据，每页间隔20
        a = str(a)
        t = threading.Thread(target=producer, args=(a, q))
        t.start()
        threads_producer.append(t)

    # 等待生产者线程全部完成
    for t in threads_producer:
        t.join()

    # 创建消费者线程
    num_consumers = 4  # 假设我们有4个消费者线程
    threads_consumer = []
    stat = time.time()
    for _ in range(num_consumers):
        c = threading.Thread(target=consumer, args=(q,))
        c.start()
        threads_consumer.append(c)
    for _ in range(num_consumers):
        q.join()
    end = time.time()
    print("Crawling completed.", end - stat)
